#!C:\Python27\python
# -*- coding:utf-8 -*-

import re
import string
import urllib2
import time
import os

class ProjectSpider(object):


	def __init__(self):
		self.ourl = "http://www.zzgcjyzx.com"
		self.curl = "http://www.zzgcjyzx.com/zzztb/gcxx/002001/002001001/"
		self.datas = []
		self.project = []
		self.page = ""
		self.longString = []
		self.firstWord = "投标人资格要求"
		self.firstHex = "\xcd\xb6\xb1\xea\xc8\xcb\xd7\xca\xb8\xf1\xd2\xaa\xc7\xf3"
		self.lastWord = "招标文件的获取" 
		self.lastHex = "\xd5\xd0\xb1\xea\xce\xc4\xbc\xfe\xb5\xc4\xbb\xf1\xc8\xa1"
		self.outp = []
		print u"工程爬虫准备就绪,准备爬取数据...".encode("GBK")
		print u"时间持续可能比较长的时间，大约需要90s，请耐心等候....".encode("GBK")
		print u"爬虫结束后，会将爬取的数据存放在D:\\lightSoftware\\final.txt文件里".encode("GBK")
		
	def getPage(self, url):
		
		
		
		try:
			page = urllib2.urlopen(url).read()
		except urllib2.URLError, e:
			if hasattr(e, "code"):
				print "The Server couldn't fulfill the request."
				print "Error Code: %s" % e.code
			elif hasattr(e, "reason"):
				print "We failed to reach a server. Please check your url and read the Reason."
				print "Reason: %s" % e.reason
		
		return page
		
	
	def getProject(self):
		
		data = self.page
		
		project_list = re.findall(r"<td width='20'><img src='/zzztb/images/dots/dot1.gif' border=0></td>(.*?)<td width='80'>", data, re.S)
		
		self.project = project_list
		#self.writeProjectSource()
		
			
	def writeProjectSource(self):
		
		f = open("D:\\lightSoftware\\source.txt", "w")
		f.writelines(self.project)
		f.close()
			
	def saveData(self):
		
		
		for index, item in enumerate(self.project):
			herf = ''.join(re.findall(r"<td><a href='(.*?)' target='_blank'", item, re.S))
			#title = ''.join(re.findall(r"target='_blank' title='(.*?)'>", item, re.S))
			title = ''.join(re.findall(r"target='_blank' title='.*?'>(.*?)</a></td>", item, re.S))
			
			#a = re.findall(r"target='_blank' title='*?'>(.*?)</a></td>", item, re.S)
			
			#print a
			lst = [herf, title]
			self.datas.append(lst)
			
			#print index
		
		#self.writeDateToFile()
	
	def listToString(self):
		#print "################################################################"
		#print self.datas
		#print "################################################################"
		for item in self.datas:
			str = '\t'.join(item)
			
			self.longString.append(str)
		
			
			
		
	
	def writeDateToFile(self):
		self.listToString()
		g = open("D:\\lightSoftware\\data.txt", "w")
		ss = '\n'.join(self.longString)
		#print ss
		g.writelines(ss)
		g.close()
			
	
	def listoutput(self):
		
		f = open("D:\\lightSoftware\\final.txt", "w")
		g = self.outp
		h = "\n\n\n".join(g)
		f.writelines(h)
		f.close()

		s = "D:\\lightSoftware\\"
		t = "D:\\lightSoftware\\final.txt"
		os.system("explorer.exe %s" % s)
		os.system("notepad.exe %s" % t)
		
	

	
	def test(self):
		
		self.page = self.getPage(self.curl)
		self.getProject()
		self.saveData()
		self.writeDateToFile()
		imax = len(self.datas)
		#print "--------------------------------------------------------------"
		for i in range(0, imax):
			#time.sleep(3)
			p = self.getProjectPage(i)
			q = self.getWordData(p)
			r = self.removeHTMLTags(q)
			#print r
			s = r.replace("&nbsp;", "")
			t = s.replace("&ldquo;", '"')
			u = t.replace("&rdquo;", '"')
			v = u.replace("&ge;", '>')
			#print v

                        dat = self.datas[i]
                        turl = dat[0]
                        title = dat[1]

                        w = title + '\n'+ self.ourl + turl + '\n' + '\n' + v
			
			self.outp.append(w)
			time.sleep(3)
			
			#print "--------------------------------------------------------------"
		
		self.listoutput()




	def getProjectPage(self, index):
		
		datas = self.datas
		data = datas[index]
		this_url = data[0]
		url = self.ourl + this_url
		page = self.getPage(url)
		#print page
		return page

	def getWordData(self, page):
		html = page
		a = re.findall(r"\xcd\xb6\xb1\xea\xc8\xcb\xd7\xca\xb8\xf1\xd2\xaa\xc7\xf3(.*?)\xd5\xd0\xb1\xea\xce\xc4\xbc\xfe\xb5\xc4\xbb\xf1\xc8\xa1", html, re.S)
		aS = ''.join(a)
		return aS
		
	def removeHTMLTags(self, data):
		html = data
		h = re.compile('</?\w+[^>]*>')
		s = h.sub('', html)
		return s
	
	"""
	def removeSomeTag(self, data, ori, des):
		dat = data
		
		a = dat.replace(des, ori)
		
		return a
		
	"""	
	












"""




0.0.2:
fix that the cant get all page bug




"""



